#include "../include/page_lib_preprocessor.hpp"

#include <iostream>
#include <fstream>
#include <sstream>
#include <memory>
#include <cmath>

using std::ifstream;
using std::ofstream;
using std::stringstream;
using std::cout;
using std::endl;
using std::shared_ptr;
using std::unique_ptr;

namespace gcj
{

PageLibPreprocessor::PageLibPreprocessor()
{ 
    /* _jieba = SplitTool::getInstance(); */
}

void PageLibPreprocessor::doProcess() {
    //预处理
    //TODO
}

void PageLibPreprocessor::readInfoFromFile() {
    //读取网页库和位置偏移库的内容
    ifstream *ifs4Page = new ifstream("../data/newRipePage.dat");
    ifstream *ifs4offset = new ifstream("../data/newOffset.dat");
    //====停止词====
    SplitTool *splitTool = SplitTool::getInstance();
    set<string> stopWords = splitTool->getStopWords();

    //需要先把网页偏移库读出来，然后根据偏移库从网页库中读网页的内容
    string tmp;
    stringstream offset;
    long docid = 0;
    int beg = 0;
    int len = 0;
    long cnt = 0;   //网页总数
    string offsetTmp;

    cout << "start load offsetLib!" << endl;
    while(getline(*ifs4offset, offsetTmp)){
        offset.str(offsetTmp);
        offset >> docid >> beg >> len;
        _offsetLib[docid] = {beg, len};

        //读取网页库中的内容
        char *pageChTmp = (char *)calloc(len + 1, sizeof(char));
        ifs4Page->read(pageChTmp, len);
        pageChTmp[len] = '\0';
        string pageStr(pageChTmp);

        shared_ptr<WebPage> wpTmp(new WebPage(pageStr));
        /* _pageLib[docid] = wpTmp; */
        _pageLib.push_back(wpTmp);  //存到网页容器中

        //回收一下资源
        offsetTmp.clear();
        offset.clear();
        free(pageChTmp);
        pageChTmp = nullptr;
        ++cnt;
    } 

    ifs4Page->close();
    ifs4offset->close();
    delete ifs4Page;
    delete ifs4offset;
}//end of readInfoFromFile

void PageLibPreprocessor::cutRedundantPages() {
    //对冗余的网页进行去重使用
    //TODO
    //直接使用第三方库simhash
}

void PageLibPreprocessor::buildInvertIndexTable() {
    readInfoFromFile();
    cout << "readInfoFromFile successful!" << endl;
    //创建倒排索引表
    //IDF = log2(N / (DF + 1))
    cout << "start build..." << endl;
    size_t cnt = _pageLib.size();   //页面总数，公式中的N
    /* cout << "N = " << cnt << endl; */
    //key是词语，value是包含该词语的页面的id和在那个页面的词频（TF）的集合
    unordered_map<string, set<pair<long, int>>> *freMap
        = new unordered_map<string, set<pair<long, int>>>();

    //获取每个页面的词频统计map，存到容器中，docid和词频
    //以及每个词语的DF
    for(auto & iter : _pageLib) {
        //需要遍历一下词频map
        long docid = iter->getDocID();
        auto &tmpMap(iter->getWordsMap());
        if(tmpMap.size() == 0) {
            continue;
        }
        /* cout << "docid = " << docid << endl; */
        /* cout << "tmpMap.size() = " << tmpMap.size() << endl; */

        for(auto &iter : tmpMap) {
            //保存每个词语的DF
            /* cout << "iter = " << iter << endl; */
            string word = iter.first;
            (*freMap)[word].insert(pair<long, int>(docid, iter.second));
            /* cout << "word: " << word << ",  " << (*freMap)[word] << endl; */
        }
    }

    //计算TF-IDF，也就是w  在这里根据词语求w
    //wsMap保存了当前页面所有词语的w，归一化要用
    unordered_map<long, unordered_map<string, double>> wsMap;
    for(auto &iter : *freMap) {
        string word = iter.first;
        int df = iter.second.size();  //包含该词语的页面的数量
        if(df == 0) {
            continue;
        }
        /* cout << word << "'s df : " << df << endl; */
        for(auto & fre : (*freMap)[word]) {
            long docid = fre.first;
            double tf = fre.second;//词语在当前页面的词频
            double tf_idf = tf * (log(cnt / (df)) / log(2));
            /* cout << "tf = " << tf << "  N = " << cnt << "  df = " << df << endl; */
            wsMap[docid][word] = tf_idf;//当前词语在当前页面的w
        }
    }

    //归一化    在这里，根据词语找到docid，并根据docid求w'
    for(auto & iter : *freMap) {
        string word = iter.first;
        for(auto & word2map : iter.second) {
            //求出当前页面中的所有词语的
            long docid = word2map.first;  //词语在那些页面中出现过
            double tfIdf = wsMap[docid][word];
            if(tfIdf == 0) {
                continue;
            }
            double sum = 0;
            for(auto & word2w : wsMap[docid]) {
                sum += word2w.second * word2w.second;
            }
            double realW = tfIdf / sqrt(sum);
            _invertIndexTable[word].insert({docid, realW});
        }
    }
    delete freMap;
    freMap = nullptr;
        
}//end of buildInvertIndexTable

void PageLibPreprocessor::storeOnDisk() {
    //将经过预处理后的网页库、位置偏移库和倒排索引库写回到磁盘
    //TODO
    //倒排索引库
    buildInvertIndexTable();    //建立倒排索引
    cout << "build successful!" << endl;
    ofstream ofs;
    cout << "start write\n";
    ofs.open("../data/invertindex.dat", std::ios::out);
    for(auto & word2w : _invertIndexTable) {
        ofs << word2w.first << "\t";
        for(auto & w : word2w.second) {
            ofs << w.first << "\t" << w.second << "\t";
        }
        ofs << "\n";
    }
    ofs.close();
}

void PageLibPreprocessor::test() {
    //这里可以读出来，但是不能根据下标读到，是WebPage成员用了引用的问题
    /* cout << "_pageLib.size() = " << _pageLib.size() << endl; */

    storeOnDisk();
    cout << "write successful" << endl;
}

}//end of namespace gcj

